<!DOCTYPE HTML>
<html lang="zh" class="light" dir="ltr">
    <head>
        <!-- Book generated using mdBook -->
        <meta charset="UTF-8">
        <title>一个监控服务的pprof 优化流程 - panda note :)</title>


        <!-- Custom HTML head -->
        
        <meta name="description" content="The note book of pandazhangs">
        <meta name="viewport" content="width=device-width, initial-scale=1">
        <meta name="theme-color" content="#ffffff">

        <link rel="icon" href="favicon.svg">
        <link rel="shortcut icon" href="favicon.png">
        <link rel="stylesheet" href="css/variables.css">
        <link rel="stylesheet" href="css/general.css">
        <link rel="stylesheet" href="css/chrome.css">
        <link rel="stylesheet" href="css/print.css" media="print">

        <!-- Fonts -->
        <link rel="stylesheet" href="FontAwesome/css/font-awesome.css">
        <link rel="stylesheet" href="fonts/fonts.css">

        <!-- Highlight.js Stylesheets -->
        <link rel="stylesheet" href="highlight.css">
        <link rel="stylesheet" href="tomorrow-night.css">
        <link rel="stylesheet" href="ayu-highlight.css">

        <!-- Custom theme stylesheets -->
        <link rel="stylesheet" href="js/whichlang.css">

        <!-- MathJax -->
        <script async src="https://cdnjs.cloudflare.com/ajax/libs/mathjax/2.7.1/MathJax.js?config=TeX-AMS-MML_HTMLorMML"></script>
    </head>
    <body class="sidebar-visible no-js">
    <div id="body-container">
        <!-- Provide site root to javascript -->
        <script>
            var path_to_root = "";
            var default_theme = window.matchMedia("(prefers-color-scheme: dark)").matches ? "navy" : "light";
        </script>

        <!-- Work around some values being stored in localStorage wrapped in quotes -->
        <script>
            try {
                var theme = localStorage.getItem('mdbook-theme');
                var sidebar = localStorage.getItem('mdbook-sidebar');

                if (theme.startsWith('"') && theme.endsWith('"')) {
                    localStorage.setItem('mdbook-theme', theme.slice(1, theme.length - 1));
                }

                if (sidebar.startsWith('"') && sidebar.endsWith('"')) {
                    localStorage.setItem('mdbook-sidebar', sidebar.slice(1, sidebar.length - 1));
                }
            } catch (e) { }
        </script>

        <!-- Set the theme before any content is loaded, prevents flash -->
        <script>
            var theme;
            try { theme = localStorage.getItem('mdbook-theme'); } catch(e) { }
            if (theme === null || theme === undefined) { theme = default_theme; }
            var html = document.querySelector('html');
            html.classList.remove('light')
            html.classList.add(theme);
            var body = document.querySelector('body');
            body.classList.remove('no-js')
            body.classList.add('js');
        </script>

        <input type="checkbox" id="sidebar-toggle-anchor" class="hidden">

        <!-- Hide / unhide sidebar before it is displayed -->
        <script>
            var body = document.querySelector('body');
            var sidebar = null;
            var sidebar_toggle = document.getElementById("sidebar-toggle-anchor");
            if (document.body.clientWidth >= 1080) {
                try { sidebar = localStorage.getItem('mdbook-sidebar'); } catch(e) { }
                sidebar = sidebar || 'visible';
            } else {
                sidebar = 'hidden';
            }
            sidebar_toggle.checked = sidebar === 'visible';
            body.classList.remove('sidebar-visible');
            body.classList.add("sidebar-" + sidebar);
        </script>

        <nav id="sidebar" class="sidebar" aria-label="Table of contents">
            <div class="sidebar-scrollbox">
                <ol class="chapter"><li class="chapter-item expanded "><a href="chapter_1.html"><strong aria-hidden="true">1.</strong> 异步日志写设计实现</a></li><li class="chapter-item expanded "><a href="chapter_2.html"><strong aria-hidden="true">2.</strong> 分布式一致性hash实现</a></li><li class="chapter-item expanded "><a href="chapter_3.html"><strong aria-hidden="true">3.</strong> 文件多格式压缩实现</a></li><li class="chapter-item expanded "><a href="snowflake.html"><strong aria-hidden="true">4.</strong> 雪花算法go 实现</a></li><li class="chapter-item expanded "><a href="gopprof.html" class="active"><strong aria-hidden="true">5.</strong> 一个监控服务的pprof 优化流程</a></li><li class="chapter-item expanded "><a href="subpush.html"><strong aria-hidden="true">6.</strong> 消息订阅推送实现</a></li><li class="chapter-item expanded "><a href="tls_h2.html"><strong aria-hidden="true">7.</strong> tls 和 h2 使用简记</a></li><li class="chapter-item expanded "><a href="grpc.html"><strong aria-hidden="true">8.</strong> gRPC 学习笔记</a></li><li class="chapter-item expanded "><a href="etcd.html"><strong aria-hidden="true">9.</strong> etcd 学习笔记</a></li><li class="chapter-item expanded "><a href="rust/rshsum.html"><strong aria-hidden="true">10.</strong> 校验和生成工具</a></li></ol>
            </div>
            <div id="sidebar-resize-handle" class="sidebar-resize-handle">
                <div class="sidebar-resize-indicator"></div>
            </div>
        </nav>

        <!-- Track and set sidebar scroll position -->
        <script>
            var sidebarScrollbox = document.querySelector('#sidebar .sidebar-scrollbox');
            sidebarScrollbox.addEventListener('click', function(e) {
                if (e.target.tagName === 'A') {
                    sessionStorage.setItem('sidebar-scroll', sidebarScrollbox.scrollTop);
                }
            }, { passive: true });
            var sidebarScrollTop = sessionStorage.getItem('sidebar-scroll');
            sessionStorage.removeItem('sidebar-scroll');
            if (sidebarScrollTop) {
                // preserve sidebar scroll position when navigating via links within sidebar
                sidebarScrollbox.scrollTop = sidebarScrollTop;
            } else {
                // scroll sidebar to current active section when navigating via "next/previous chapter" buttons
                var activeSection = document.querySelector('#sidebar .active');
                if (activeSection) {
                    activeSection.scrollIntoView({ block: 'center' });
                }
            }
        </script>

        <div id="page-wrapper" class="page-wrapper">

            <div class="page">
                                <div id="menu-bar-hover-placeholder"></div>
                <div id="menu-bar" class="menu-bar sticky">
                    <div class="left-buttons">
                        <label id="sidebar-toggle" class="icon-button" for="sidebar-toggle-anchor" title="Toggle Table of Contents" aria-label="Toggle Table of Contents" aria-controls="sidebar">
                            <i class="fa fa-bars"></i>
                        </label>
                        <button id="theme-toggle" class="icon-button" type="button" title="Change theme" aria-label="Change theme" aria-haspopup="true" aria-expanded="false" aria-controls="theme-list">
                            <i class="fa fa-paint-brush"></i>
                        </button>
                        <ul id="theme-list" class="theme-popup" aria-label="Themes" role="menu">
                            <li role="none"><button role="menuitem" class="theme" id="light">Light</button></li>
                            <li role="none"><button role="menuitem" class="theme" id="rust">Rust</button></li>
                            <li role="none"><button role="menuitem" class="theme" id="coal">Coal</button></li>
                            <li role="none"><button role="menuitem" class="theme" id="navy">Navy</button></li>
                            <li role="none"><button role="menuitem" class="theme" id="ayu">Ayu</button></li>
                        </ul>
                        <button id="search-toggle" class="icon-button" type="button" title="Search. (Shortkey: s)" aria-label="Toggle Searchbar" aria-expanded="false" aria-keyshortcuts="S" aria-controls="searchbar">
                            <i class="fa fa-search"></i>
                        </button>
                    </div>

                    <h1 class="menu-title">panda note :)</h1>

                    <div class="right-buttons">
                        <a href="print.html" title="Print this book" aria-label="Print this book">
                            <i id="print-button" class="fa fa-print"></i>
                        </a>

                    </div>
                </div>

                <div id="search-wrapper" class="hidden">
                    <form id="searchbar-outer" class="searchbar-outer">
                        <input type="search" id="searchbar" name="searchbar" placeholder="Search this book ..." aria-controls="searchresults-outer" aria-describedby="searchresults-header">
                    </form>
                    <div id="searchresults-outer" class="searchresults-outer hidden">
                        <div id="searchresults-header" class="searchresults-header"></div>
                        <ul id="searchresults">
                        </ul>
                    </div>
                </div>

                <!-- Apply ARIA attributes after the sidebar and the sidebar toggle button are added to the DOM -->
                <script>
                    document.getElementById('sidebar-toggle').setAttribute('aria-expanded', sidebar === 'visible');
                    document.getElementById('sidebar').setAttribute('aria-hidden', sidebar !== 'visible');
                    Array.from(document.querySelectorAll('#sidebar a')).forEach(function(link) {
                        link.setAttribute('tabIndex', sidebar === 'visible' ? 0 : -1);
                    });
                </script>

                <div id="content" class="content">
                    <main>
                        <h1 id="prometheus-exporter-pprof-优化"><a class="header" href="#prometheus-exporter-pprof-优化">Prometheus exporter pprof 优化</a></h1>
<hr />
<blockquote>
<p>这个工程是 基于 Prometheus client-go 的库来开发的，这个库的主要流程是将 collector 的数据用http server的形式通过metrics 路由交出去，背景状况是现在这个收集器的收集间隔很长基本上是6个小时更新一次，但是负责抓取这些指标数据的服务leader 又不同意改成6小时的大跨度，所以之前的措施是把硬件采集的数据缓存起来，但是更新后的版本cpu占用率和响应时间都没太大优化，需要定位这个问题</p>
</blockquote>
<h2 id="怎么采集后台运行进程性能数据"><a class="header" href="#怎么采集后台运行进程性能数据">怎么采集后台运行进程性能数据</a></h2>
<ol>
<li>使用<code>ps -ef | grep xxx</code> 来获取进程号pid
<ol>
<li><code> pidstat -u -p pid  15 4</code> 来采集进程的cpu 占用率</li>
</ol>
</li>
<li>top 按t 和m 来切换 cpu 和 内存排序</li>
<li>go 使用  _ "net/http/pprof" 然后非http 网络服务就再增加一个拉起http 服务的几行代码
<ol>
<li>使用curl -o cpupgo.out http://your_address:your_port/debug/pprof/profile?seconds=60 来采样一分钟的运行数据</li>
<li><code>go tool pprof -http=:9000 cpupgo.out </code> 使用 pprof 工具开启一个网络服务在web网页上查看性能采样</li>
</ol>
</li>
<li>使用 perf 工具来采样 基本上会用到 record 和 report 这些，然后转成火山图来分析</li>
</ol>
<h2 id="问题追踪"><a class="header" href="#问题追踪">问题追踪</a></h2>
<p><strong>问题1.</strong> 缓存为什么没有生效(降低延迟减少耗时操作)
通过go 的pprof 对后台运行的服务采样后发现大部分cpu 时间在生成Metric 相关的结构体上，同时缓存的数据格式是json，取json 数据会用到仿射，这使得组装Metric 的过程中充斥着大量的耗时操作，于是选择将json 数据缓存改成 Metric 数据缓冲，和时效时间戳一起封装成一个抽象的容器，在未过期时会将，slice 里面所有的Metric 通过管道发送出去，过期时会将slice 长度重置，发送Metric 的同时将其append 到slice 里面缓存，总结来说，缓存生效了，但是又没完全覆盖到所有耗时操作上。</p>
<p>改进后的缓存实现，在应用中遇到了新的问题，缓存应用后没被触发？</p>
<p><strong>问题2</strong>.  缓存为什么没被触发？
复盘对比了两种缓存机制和Collect 方法被调用的过程时发现，Collect 中声明的对象在每次调用时重新创建的，之前json 缓存是用的全局变量，所以创建前后用的都是一个缓存；这里将新的缓存实现也没大改，给新建的这些对象实例也做个全局缓存，没过期失效前这些实例就不会被重新创建，减少了一些再分配构建的过程，通过预留的 cache stat handler 可以看到缓存除了初次和过期时未命中外，其余时刻缓存全命中，符合预期</p>
<p>新的全局缓冲实现生效了，将cpu 占用率降低到原先的30%，同时内存占用差别不大，但剩下的30%还能不能继续优化呢？</p>
<h2 id="终极优化方案"><a class="header" href="#终极优化方案">终极优化方案</a></h2>
<p>通过对 Prometheus client go 的源码阅读，确定了相应http 响应的整个构造流程，脑中浮现了一个比较极端的想法，缓冲响应；</p>
<p>这个适合用来缓存响应内容在一段时间内不会发生改动的 http handler 接口对象</p>
<pre><code class="language-go">/* get resp cache code  start  */
type respCacheWriter struct {
	header     http.Header
    expireat time.Time
	statuscode int
	buf        []byte
    update     bool
}
func newRespCacheWriter() *respCacheWriter {
    return &amp;respCacheWriter{
        header: make(http.Header, 3),
		buf:    make([]byte, 0, 1024*4),
	}
}
func (r *respCacheWriter) NotExpire() bool {
    return time.Now().Before(r.expireat)&amp;&amp;!r.update
}
func (r *respCacheWriter) Update(interval time.Duration) {
     r.expireat = time.Now().Add(interval)
     r.update = false
}
func (r *respCacheWriter) Header() http.Header {
	return r.header
}
func (r *respCacheWriter) SetUpdate() http.Header {
	return r.update = true
}

func (r *respCacheWriter) WriteHeader(statusCode int) {
	r.statuscode = statusCode
}

func (r *respCacheWriter) Write(p []byte) (int, error) {
	if p == nil {
		return 0, fmt.Errorf("Write []byte length should not be zero")
	}
	r.buf = append(r.buf, p...)
	return len(r.buf), nil
}

type GetRespCache struct {
	
	interval time.Duration
	cache    map[string]*respCacheWriter
	next     http.Handler
}

func NewGetRespCache(i time.Duration,next http.Handler)GetRespCache{
	return GetRespCache{ 
		interval: i,
		cache: make(map[string]*respCacheWriter,1),
		next: next,
	}
}

// 这个方法其实也可以 转成私有的，但是不会修改cache 状态所以无所谓

func (g *GetRespCache)UpdateCache()bool{
    key := fmt.Sprintf("key%v", r.URL.Query())
    if c,ok:=g.cache[key];key&amp;&amp;c!=nil{
        c.SetUpdate()
    }
}

func (g *GetRespCache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	key := fmt.Sprintf("key%v", r.URL.Query())
	if val, ok := g.cache[key]; ok  &amp;&amp; val != nil&amp;&amp;val.NotExpire() {
		g.generateResp(val, w)
	} else {
		// 调用 write 方法时 缓存响应的 header
		resp := newRespCacheWriter()
		g.next.ServeHTTP(resp, r)
		g.generateResp(resp, w)
		g.update(key, resp)

	}
}
func (g *GetRespCache) update(key string, resp *respCacheWriter) {
    // 这里会更新过期时间和下一个响应状态
	resp.Update(g.interval)
    g.cache[key] = resp
}
func (*GetRespCache) generateResp(val *respCacheWriter, w http.ResponseWriter) {
	for k, s := range val.header {
		for _, v := range s {
			w.Header().Set(k, v)
		}
	}
	w.WriteHeader(val.statuscode)
	w.Write(val.buf)
}
/* get resp cache code  end  */

</code></pre>
<p>可以看到其实就是通过中间的代理接口，将被代理的handler 函数的修改缓存起来，根据get 请求的query值来做hash 返回响应的；
过期的时效间隔这里倒是比较粗，用的是同一个过期间隔；</p>
<p>这个get 缓存方案是我最看好的:</p>
<ul>
<li>第一点基本上是即插即用，迁移性好兼容性好，</li>
<li>第二点是性能更好，缓存占用少，还剩去了内部handler 处理的时间</li>
</ul>
<p>这个之所以能用在这个场景上，其实是需求造成的，抓取端不改动，数据供应端又允许缓存；所以这个getcache 的方案理论上有奇效，但是最终还是没应用上这个，确定是按照问题2解决后的实现方案来。</p>

                    </main>

                    <nav class="nav-wrapper" aria-label="Page navigation">
                        <!-- Mobile navigation buttons -->
                            <a rel="prev" href="snowflake.html" class="mobile-nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
                                <i class="fa fa-angle-left"></i>
                            </a>

                            <a rel="next prefetch" href="subpush.html" class="mobile-nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
                                <i class="fa fa-angle-right"></i>
                            </a>

                        <div style="clear: both"></div>
                    </nav>
                </div>
            </div>

            <nav class="nav-wide-wrapper" aria-label="Page navigation">
                    <a rel="prev" href="snowflake.html" class="nav-chapters previous" title="Previous chapter" aria-label="Previous chapter" aria-keyshortcuts="Left">
                        <i class="fa fa-angle-left"></i>
                    </a>

                    <a rel="next prefetch" href="subpush.html" class="nav-chapters next" title="Next chapter" aria-label="Next chapter" aria-keyshortcuts="Right">
                        <i class="fa fa-angle-right"></i>
                    </a>
            </nav>

        </div>

        <!-- Livereload script (if served using the cli tool) -->
        <script>
            const wsProtocol = location.protocol === 'https:' ? 'wss:' : 'ws:';
            const wsAddress = wsProtocol + "//" + location.host + "/" + "__livereload";
            const socket = new WebSocket(wsAddress);
            socket.onmessage = function (event) {
                if (event.data === "reload") {
                    socket.close();
                    location.reload();
                }
            };

            window.onbeforeunload = function() {
                socket.close();
            }
        </script>



        <script>
            window.playground_copyable = true;
        </script>


        <script src="elasticlunr.min.js"></script>
        <script src="mark.min.js"></script>
        <script src="searcher.js"></script>

        <script src="clipboard.min.js"></script>
        <script src="highlight.js"></script>
        <script src="book.js"></script>

        <!-- Custom JS scripts -->
        <script src="js/fzf.umd.js"></script>
        <script src="js/elasticlunr.js"></script>
        <script src="js/import-html.js"></script>
        <script src="js/whichlang.js"></script>


    </div>
    </body>
</html>
